# -*- coding: utf-8 -*-
#
# Copyright (c) 2011,2012 RMIT e-Research Office
#   (RMIT University, Australia)
# All rights reserved.
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#    *  Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#    *  Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#    *  Neither the name of RMIT University nor the
#       names of its contributors may be used to endorse or promote products
#       derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
"""
.. module:: views
   :synopsis: the views for the application

.. moduleauthor:: Ian Thomas <Ian.Edward.Thomas@rmit.edu.au>


"""
import logging
import os
import time
import ast
from datetime import date                     
import datetime
from itertools import chain
from urllib2 import Request, URLError, HTTPError


from django.utils.datastructures import SortedDict  
from django.conf import settings
from django.shortcuts import render_to_response
from django.shortcuts import get_object_or_404
from django.shortcuts import redirect
from django.template import RequestContext
from django.views.generic.base import TemplateView
from django.views.generic.list import ListView 
from django.utils import simplejson 
from django import forms
from django.core.exceptions import ObjectDoesNotExist
from django.utils.http import urlquote_plus
from django.core.urlresolvers import reverse
from django.http import HttpResponseBadRequest, HttpResponseForbidden, HttpResponseServerError
from django.core.paginator import Paginator, InvalidPage, EmptyPage, PageNotAnInteger
from django.http import Http404
from django.contrib import messages
from django.forms.formsets import formset_factory

from smra.middleware.http import Http403
from smra.smra_portal.QueryExpression import DESCLABEL
from smra.smra_portal.QueryExpression import CONTAINS_OP
from smra.smra_portal.QueryExpression import SUBQUERYLABEL 
from smra.smra_portal.QueryExpression import QueryExpression
from smra.smra_portal.QueryExpression import QueryExpressionError
from smra.smra_portal.QueryExpression import Disjunction
from smra.smra_portal.QueryExpression import parse_disjunction
from smra.smra_portal.QueryExpression import parse_query
from smra.smra_portal.forms import DescForm
from smra.smra_portal.forms import QueryForm
from smra.smra_portal.forms import VirtForm
from smra.smra_portal.forms import SmallVirtForm
from smra.smra_portal.models import MediaObjectParameterSet
from smra.smra_portal.models import MediaObject
from smra.smra_portal.models import MediaObjectParameter
from smra.smra_portal.models import DeadLink
from smra.smra_portal.models import ParameterName
from smra.smra_portal.models import Schema
from smra.smra_portal.models import UserProfile
from smra.smra_portal.models import VirtualCollection
from smra.smra_portal.models import Subquery
from smra.smra_portal import stringbank
from smra.smra_portal.models import VirtualCollectionParameter
from smra.smra_portal.models import VirtualCollectionParameterSet
from smra.smra_portal.models import VirtualCollectionHarvest
from smra.smra_portal.models import PartyRecord
    

    

logger = logging.getLogger(__name__)

def home(request):
    """ View the home page of the site
    
        :param request: the web request
        :type request: :class:`django.http.HttpRequest`
     """
    return render_to_response('smra_portal/home.html', {},
                              context_instance=RequestContext(request))


def mediaobject_details(request, pk):
    """ View detail page for a selected media object, 
        including all metadata
    
        :param request: the web request
        :type request: :class:`django.http.HttpRequest`
        :param pk: the key for the media object
        :type pk: primary key
    """ 
    if request.user.is_authenticated():  
        pass
    
    obj = get_object_or_404(MediaObject, pk=pk)
    logger.debug("obj=" + obj.description)
    parameter_sets = MediaObjectParameterSet.objects.filter(media_object=obj)
    logger.debug("parameter_sets=" + repr(parameter_sets))
    mediaobject_parameters = MediaObjectParameter.objects.filter(
                                                paramset__in =parameter_sets)
    metadata = []
    for paramset in parameter_sets:
        logger.debug("paramset=" + repr(paramset)) 
        params = []
        for param in mediaobject_parameters:
            logger.debug("param=" + repr(param))
            if param.paramset.pk == paramset.pk:
                logger.debug("param.paramset.pk=%s paramset.pk=%s"
                             %(param.paramset.pk, paramset.pk))
                logger.debug("parm.name.name=%s" % param.name.name)
                logger.debug("param.name.type=%s" % param.name.type)
                logger.debug("param.value=%s" % param.value)
                if param.name.type == ParameterName.NUMERIC:
                    try:
                        val = int(param.value)
                    except ValueError:
                        val = param.value # TODO: should give warning that input is invalid
                elif param.name.type == ParameterName.DATE:
                    val = str(param.value)
                    #from dateutil import parser
                    #try:
                    #    val = parser.parse(param.value).date()
                    #except ValueError:
                    #    val = param.value  #TODO: should give warning that input is invalid
                elif param.name.type == ParameterName.YEAR:
                    try:
                        val = int(param.value)
                    except ValueError:
                        val = param.value # TODO: should give warning that input is invalid
                else:
                    val = param.value
                    # FIXME: can't call str on this because it may be unicode
                    val = param.value
                logger.debug("val=%s" % val)
                params.append((param.name.name,
                               ParameterName.TYPES[param.name.type][1],
                               val))
                #params[param.name.name] = str(param.value) 
        metadata.append((paramset.schema.description,params))
    logger.debug("metadata=%s" % metadata)
    context = {}
    context['description'] = obj.description 
    context['metadata'] = metadata
    deadlinks_num = DeadLink.objects.count()
    context['deadlinks_num'] = str(deadlinks_num)
    return render_to_response('smra_portal/mediaobject.html',
                            context, 
                            context_instance=RequestContext(request))


_invalid_query_message = 'There was some thing wrong with this \
                    query. Enter set of keywords. Put "-" at beginning of \
                    keyword to exclude it fperform_searcharch. Put " around keywords \
                     to select exact strings. \
                     For advanced searches, prefix the string with VC: and \
                     use boolean expressions. \
                     Here is an example: \
                     VC:description=="Campion" & ~"Rating"=="MA"'       

               
def _make_desc_query(desc_query):
    """ Returns a list of terms corresponding basic query
    
      :param desc_query: basic description-based query
      :type: desc_query: str 
      :returns: list of search term strings
    
    """
    logger.debug("desc_query = %s" % desc_query)
    qe = parse_query(desc_query)       
    if qe:
        if desc_query[0:3] == "VC:":               
            bool_query = [x.strip() for x in desc_query[3:].split('&')]
        else:
            keywords = []
            while '"' in desc_query:
                begin_quote = desc_query.find('"')
                end_quote = desc_query.find('"', begin_quote+1)
                quoted_keys = desc_query[begin_quote:end_quote+1]
                if begin_quote > 0 and desc_query[begin_quote-1] == '-':
                    keywords.append("-" + quoted_keys.strip('"'))
                    desc_query = desc_query.replace("-" + quoted_keys, ' ')
                else:
                    keywords.append(quoted_keys.strip('"'))
                    desc_query = desc_query.replace(quoted_keys, ' ')
            keywords.extend(desc_query.split())
            # TODO: handle "description" parameter field and 
            # "description" object field confusion        
            bool_query = []
            for word in keywords:    
                if word[0] == '-': 
                    bool_query.append('~%s%s"%s"' % (DESCLABEL, CONTAINS_OP,
                                                     word[1:]))
                else:
                    bool_query.append('%s%s"%s"'%(DESCLABEL,CONTAINS_OP,
                                                  word))                  
    else:
        bool_query=[]
    return bool_query       

    
def _handle_page_length(request,profile):
    """ Get the new results per page and set profile 
    
        :param request: the web request
        :type request: :class:`django.http.HttpRequest`
        :param profile: the user profile
        :type pk: :class:`smra.smra_portal.models.UserProfile`
    """
        
    if 'pagelength' in request.POST:
        plen = request.POST['pagelength']
        try:
            pagelen = int(plen)
        except ValueError:
            pagelen = settings.DEFAULT_RESULTS_PER_PAGE
        if pagelen <= 0:
            pagelen = settings.DEFAULT_RESULTS_PER_PAGE
        profile.results_per_page = pagelen
        profile.save()
        
            
def _handle_page_buttons(request):
    """ Get the requested page number from previous/next buttons
        
        :param request: the web request
        :type request: :class:`django.http.HttpRequest`  
        :returns: page number
        :rtype: int
    """
    # FIXME: this is inefficent as we redo search each time, plus
    # it duplicate into history list.  There is also a problem when
    # we later want to select search results items spread over multiple pages
    page=1
    if 'next'  in request.POST:        
        pagefield = request.POST['nextpage']
        try:
            page = int(pagefield)
        except ValueError:
            page = 1
    if 'previous' in request.POST:
        pagefield = request.POST['previouspage']
        try:
            page = int(pagefield)
        except ValueError:
            page = 1
    if 'first' in request.POST:
        pagefield = request.POST['firstpage']
        try:
            page = int(pagefield)
        except ValueError:
            page = 1
    if 'last' in request.POST:
        pagefield = request.POST['lastpage']
        try:
            page = int(pagefield)
        except ValueError:
            page = 1        
    return page


def _handle_desc_form(request,new_query):
    """ Get the contents of the description fields from the request form
        and add as term to new_query and return
    
        :param request: the web request
        :type request: :class:`django.http.HttpRequest`      
        :param new_query: a list of search expressions
        :type new_query: list
        :returns: tuple of list query expression strings and form
        
    """
    # Create query corresponding to description field form
    form = DescForm(request.POST)
    if form.is_valid():
        cleaned = form.cleaned_data #TODO
        desc_query = cleaned['desc_field']
        #logger.debug("post=%s" % request.POST)                                                                    
        new_query.extend(_make_desc_query(desc_query))
        logger.debug("new desc_query = %s" %new_query)
    else:        
        logger.error("desc form has errors %s" % form.errors)
    return (new_query,form)
        
        
def _handle_media_form(request,new_query):
    """ Add query corresponding to media type field
    
    :param request: the web request
    :type request: :class:`django.http.HttpRequest`
    :param new_query: a list of search expressions
    :type new_query: list
    :returns: the updated new_query
          
    """
    media_form =  _make_media_form(req=request.POST)
    if media_form.is_valid():
        choices = media_form.cleaned_data['mediachoices']
        logger.debug("mediachoices=%s" % choices)         
        for choice in choices:
            new_query.append('mediatype%s"%s"' % (CONTAINS_OP,choice))
        not_choices = media_form.cleaned_data['notmediachoices']
        logger.debug("notmediachoices=%s" % not_choices)
        for choice in not_choices:
            new_query.append('~mediatype%s"%s"' % (CONTAINS_OP,choice))        
        logger.debug("bound_media_form= %s" % media_form)
        logger.debug("choice=%s" % choices)
    else:
        logger.debug("media_form is invalid %s" % media_form.errors)
    return new_query
    
    
def _handle_results_form(request):
    """ Process the search results form
     
        :param request: the web request
        :type request: :class:`django.http.HttpRequest`
        :returns: the new query from the form
        
    """
    logger.debug("handling results form")
        
    logger.debug("request=%s" % request.POST)
    
    # NOTE: Because the search results list is different everytime, 
    # we cannot construct the original form to bind POST to it. Therefore,
    # we have to deal with the POST data directly.  This is a little slow.
    new_query=[]
    from collections import defaultdict
    is_neg = defaultdict(bool)
    
    for key in request.POST:
        parts = key.split("/")
        if len(parts) ==3 and parts[0] == "SR" and parts[2] == "NOT":    
            val = request.POST[key]
            if val == "on":                
                logger.debug("neg key=%s" % key)
                is_neg[val]= True
            
    for key in request.POST:
        parts = key.split("/")
        if len(parts) ==3 and parts[0] == "SR" and parts[2] == "INC":    
            val = request.POST[key]
            if val == "on":                
                logger.debug("key=%s" % key)
                # FIXME: We should really not use CONTAINS_OP, as ID is
                # really an exact match in this case.
                if is_neg[val]:
                    new_query.append(str('VC:~id%s"%s"' % (CONTAINS_OP,parts[1])))      
                else:
                    new_query.append(str('VC:id%s"%s"' % (CONTAINS_OP,parts[1])))      
   
#    (_,results_form) = _make_results_form(req=request.POST)
                
#    if results_form.is_valid():
#        logger.debug("results_form=%s" % results_form)
#        for key,val in results_form.cleaned_data.items():
#            logger.debug("key=%s,val=%s" %(key,val))
#            parts = key.split("/")
#            if len(parts) ==3 and parts[0] == "SR" and parts[2] == "INC":
#                logger.debug("key=%s" % key)
#                new_query.append('id.equals."%s"' % val)                
#    else:
#        logger.debug("results form is invalid %s" % results_form.errors)
    return new_query


def _handle_metadata_forms(request,new_query):
    """ Process the metadata forms
        
        :param request: the web request
        :type request: :class:`django.http.HttpRequest`
        :param new_query: a list of search expressions
        :type new_query: list
        :returns: the new query from the form
        
    """
    # Add query corresponding to the meta data fields
    logger.debug("handling metadata forms")
    schema_forms = _make_metadata_forms(req=request.POST)
    logger.debug("schema_forms=%s" % schema_forms)       
    for namespace,name,description,form_length,form in schema_forms:
        logger.debug(" meta dat forms %s %s %s %s" % (namespace, name,
                                                      description, form))            
        if form.is_valid(): 
            # logger.debug("metadata_form = %s" % form.cleaned_data)
            
            # find out which forms have been negated
            # Important that we process all negation fields before the associated
            # fields because the order may be incorrect.
            negated = {}
            field_type = {}
            for data in form.cleaned_data:
                parts = data.split("/")
                if len(parts) == 3 and parts[0] == 'NOT':
                    if form.cleaned_data[data]:
                        negated[(parts[1],parts[2])] = True 
                if len(parts) == 3 and parts[0] == 'TYPE':
                    if form.cleaned_data[data]:
                        field_type[(parts[1],parts[2])] = parts[3] 
                                
            for data in form.cleaned_data:
                logger.debug("%s=%s" % (data,form.cleaned_data[data]))
                parts = data.split("/")  
                logger.debug("parts = %s" % parts)
                key = form.cleaned_data[data]
                
                if key != False and key != None and key != "": # FIXME: handle empty values correctly                      
                    if len(parts) == 3:                        
                        pass
                    elif len(parts) == 2:
#                        field_value = str(key)
#                        if (parts[0], parts[1]) in field_type:
#                            val_type = 0
#                            try:
#                                val_type = int(field_type[(parts[0],parts[1])])
#                            except ValueError:
#                                val_type = 0
                        
                        # FIXME: for Dates we could do validation parsing here 
                        
                        # We assume only STRING/VALUES types allowed    
                        #if type(key) is int: # Assumes integerfield data                            
                        #    field_value = "%s" % key
                        #else:
                        field_value = '"%s"' % key                       
                        logger.debug("field_value=%s" % field_value)
                        if (parts[0],parts[1]) in negated:                                                    
                            new_query.append('~%s/%s.contains.%s' % (
                                                                 parts[0],
                                                                 parts[1],
                                                                 field_value))
                        else:
                            new_query.append('%s/%s.contains.%s' % (
                                                                parts[0],
                                                                parts[1],
                                                                field_value))
                    else:
                        # would be mediatype or description?
                        pass
        else:
            #FIXME: we are not handling validation errors very well, as
            #error message are ignored
            logger.debug("form is invalid")
            messages.error(request,"Invalid form data %s" % form.errors)
            
    
    return new_query
        
        
def _handle_del_subquery_button(request,subqueries,profile):
    """ Process the deletion of subquery buttons in the page
        :param request: the web request
        :type request: :class:`django.http.HttpRequest`
        :param subqueries: existing subqueries to populate form
        :param profile: the user profile
        :type pk: :class:`smra.smra_portal.models.UserProfile`
    
    """
    if 'deletesubquery' in request.POST:
        logger.debug("deletesubquery")
        (subquery_form,_) = _make_subquery_form(objs=subqueries,
                                                req=request.POST)
        queries_to_del = []
        if subquery_form.is_valid():
            cleaned = subquery_form.cleaned_data
            for field in cleaned:
                logger.debug("field=%s" % field)
                fieldkey_parts = field.split('/')
                if (fieldkey_parts[0] == 'SQ' and fieldkey_parts[2] == 'DEL'):
                    if cleaned[field]:
                        queries_to_del.append(fieldkey_parts[1])
            logger.debug("queries_to_del=%s" % queries_to_del)
            Subquery.objects.filter(owned_by=profile,
                                    pk__in=queries_to_del).delete() 
            messages.success(
                         request,
                         "Subqueries %s have been deleted" % queries_to_del)
        else:
            logger.debug("invalid subquery delete: %s" % subquery_form.errors)
 
 
def _handle_inc_subquery_button(request,subqueries,profile,new_query):
    """ Process the inclusion of subquery buttons in the page
        
        :param request: the web request
        :type request: :class:`django.http.HttpRequest`
        :param subqueries: existing subqueries to populate form
        :param profile: the user profile
        :param new_query: a list of search expressions
        :type new_query: list
        :type pk: :class:`smra.smra_portal.models.UserProfile`
        
    
    """
    logger.debug("incsubquery")
    (subquery_form,_) = _make_subquery_form(objs=subqueries,
                                            req=request.POST)
    if subquery_form.is_valid():
        cleaned = subquery_form.cleaned_data
        for field in cleaned:
            logger.debug("field=%s" % field)
            fieldkey_parts = field.split('/')
            if (fieldkey_parts[0] == 'SQ' and fieldkey_parts[2] == 'INC'):
                logger.debug("cleaned[field] = %s" %cleaned[field])
                if cleaned[field]:
                    new_query.append('VC:%s%s"%s"' 
                                     % (SUBQUERYLABEL,
                                        CONTAINS_OP,
                                        fieldkey_parts[1]))                    
                    
                    
        logger.debug("new_query=%s" % new_query)
    else:
        logger.debug("invalid subquery inc: %s" % subquery_form.errors)
    return new_query
 
 
def _remove_term_duplicates(query):
    # remove duplicates (will reorder)
    # FIXME: Cannot reorder terms as use of AND/OR could change meaning
    if query:
        query.sort()
        last = query[-1]
        for i in range(len(query)-2, -1, -1):
            if last==query[i]: del query[i]
            else: last=query[i]            
    return query
       

def _handle_search_post(request,historylist,profile,virts,subqueries,currentvirt):    
    logger.debug("in POST")
    
    
#    pagefield = request.GET.get('page')
#    
#    try:
#        page = int(pagefield)
#    except ValueError:
#        page = 1

    page = _handle_page_buttons(request)
    
    if profile:
        # If we are not authenticated then can't change page length
        _handle_page_length(request,profile)
    
    # Build up a disjunction which is a list of query expressions implicitly 
    # OR-ed together.
    disjunction = []
    
    conjunction = []
    try:
        conjunction,form = _handle_desc_form(request,conjunction)
    except QueryExpressionError,e:
        error_message= "invalid description field"
               
        # We do not have enough informaton to render the GET template,
        # so we simply redirect back to blank page
        # FIXME: allow error_message to be sent through too.
        if currentvirt:
            return redirect(reverse('virt',
                                args=[currentvirt.pk])  

                                  + "?s=" + urlquote_plus("['VC:']"))                                
        else:
            return redirect(reverse('virt',
                                args=[0])  

                                  + "?s=" + urlquote_plus("['VC:']"))          
    conjunction = _handle_media_form(request,conjunction)        
    conjunction = _handle_metadata_forms(request,conjunction)  
    conjunction =  str("VC:" + ' & '.join(conjunction))
    disjunction.append(conjunction)
    
    results_disj = _handle_results_form(request)
    disjunction.extend(results_disj)
                       
    if currentvirt:  
        disjunction.extend(_handle_inc_subquery_button(request, subqueries,
                                                   profile, []))
                            
    logger.debug("disjunction= %s" % disjunction)
    
    #new_query = _remove_term_duplicates(new_query)
   
    
        
    if 'browse' in request.POST:
        return redirect(reverse('browse'))
                   
      
    # if user pressed save button, then parse desc_query, then save 
    # valid desc_query as virtual collection, then rediret to GET
    # to actually do the perform_search.  
    if currentvirt:
        
        _handle_del_subquery_button(request,subqueries,profile)

                      
        if 'save' in request.POST:
            logger.debug("ready to save")      
            if disjunction:
              
                currentvirt.definition=disjunction
                currentvirt.save()              
                logger.debug("saving virt")
                messages.success(request,'Your Virtual Collection definition has been saved')
    
            else:
                # FIXME: this is a silent failure
                messages.error(request,"There was an error in your query")
                return redirect(reverse('virt',args=[currentvirt.pk]) +
                                 "?s=" + urlquote_plus("['VC:']"))                                
    
    
            
        
        if 'savesubquery' in request.POST:
            if disjunction:
                if 'subqueryname' in request.POST:
                    name = request.POST['subqueryname']
                else:
                    name = ''
                subquery = Subquery(name=name,
                                    owned_by=profile,
                                    part_of=currentvirt,
                                    query=[str(conjunction)])    
                subquery.save()              
                logger.debug("saving subquery %s" % subquery)    
            else:
                historylisttuple = [(' | '.join(ast.literal_eval(x)),urlquote_plus(x)) for x in historylist]
                messages.error(request,"There was an error in your query")
                return redirect(reverse('virt',
                                    args=[currentvirt.pk])  
    
                                      + "?s=" + urlquote_plus("['VC:']"))                                
    
            
        if 'savelist' in request.POST:
            if results_disj:
                
                if 'listname' in request.POST:
                    name = request.POST['listname']
                else:
                    name = ''
                logger.debug("results_disj = %s" %results_disj)
                subquery = Subquery(name=name,
                                    owned_by=profile,
                                    part_of=currentvirt,
                                    query=str(results_disj))    
                subquery.save()              
                logger.debug("saving sublist")    
            else:
                messages.error(request,"There was an error in your query")
                historylisttuple = [(' | '.join(ast.literal_eval(x)),urlquote_plus(x)) for x in historylist]
                return redirect(reverse('virt',
                                    args=[currentvirt.pk])  
    
                                      + "?s=" + urlquote_plus("['VC:']"))                                

     
    #import pdb; pdb.set_trace()          
             
    if page != 1:
        return redirect(reverse('virt',
                                 args=[currentvirt.pk if currentvirt else 0])
                         + '?page=%s&s=%s' %(page,
                                              urlquote_plus(repr(disjunction))))
    else:                                
        return redirect(reverse('virt',
                                args=[currentvirt.pk if currentvirt else 0])  
                        + '?s=%s' %urlquote_plus(repr(disjunction)))                                

          
def _handle_search_get(request, historylist, profile, virts,subqueries,currentvirt):
    """ Handle return of search results form a query string"""
    logger.debug("in GET")    
    error_message = None
    query_form = QueryForm(request.GET)
    if query_form.is_valid():
        cleaned = query_form.cleaned_data #TODO:implement
        desc_query = cleaned['s']        
        if not desc_query:
            # load up saved virtual collection definition      
            if not currentvirt:
                desc_query=str(["VC:"])
            else:
                desc_query = currentvirt.definition
                if not desc_query:
                    # brand new virt
                    desc_query=str(["VC:"])            
        disjunction = Disjunction()
        objs = MediaObject.objects.none()
        try:   
            disjunction = Disjunction(parse_disjunction(desc_query))
            logger.debug("disjunction is %s" % str(disjunction))
            objs = disjunction.get_objects_for_disj(currentvirt)            
        except QueryExpressionError:            
            messages.error(request,"There was an error in your query")
            error_message = True                     
          
        
        if profile:
            results_per_page = profile.results_per_page
        else:
            results_per_page = settings.DEFAULT_RESULTS_PER_PAGE
            
        if results_per_page <= 0:
            results_per_page =  settings.DEFAULT_RESULTS_PER_PAGE
        
        paginator = Paginator(objs,results_per_page)
        
        try:
            page = int(request.GET.get('page', '1'))
        except ValueError:
            page = 1 
       
        
        # If page request (9999) is out of range, deliver last page of results.
        try:
            objs = paginator.page(page)
        except (EmptyPage, InvalidPage):
            objs = paginator.page(paginator.num_pages)


        # Add to history                                   
        if page == 1 and not error_message:
            logger.debug(historylist)
            d = datetime.datetime.now()                        
            historylist.insert(0,(d.strftime("%c"),str(disjunction)))                
            historylist = historylist[0:settings.NUM_PREVIOUS_SEARCHES]                            
            logger.debug(historylist)

                
        request.session['history'] = simplejson.dumps(historylist, 
                                                      indent=4)                 
        
    else:
        messages.error(request,"There was an error in your query")
        paginator = Paginator([],settings.RESULTS_PER_PAGE)
        objs = []                                        
    
    # Populate the fields
    desc_form = _make_desc_form(disj=disjunction)
    schema_forms = _make_metadata_forms(disj=disjunction)      
    media_form = _make_media_form(disj=disjunction)
    results_form, results_form_data, results_count = _make_results_form(
                                                    objs=objs.object_list)
    (subquery_form,subquery_formdata) = _make_subquery_form(disj=disjunction,
                                                objs=subqueries)
    historylisttuple = [(') OR ('.join([y[3:] for y in ast.literal_eval(x[1])]),urlquote_plus(x[1]),x[0]) for x in historylist]
    #historylisttuple = [(x, urlquote_plus(x)) for x in historylist]
    virtlisttuple = [(x.name, urlquote_plus(x.name))  for x in virts]     
    logger.debug("virts=%s" % virts)
    logger.debug("historylist=%s" %historylist)
    deadlinks_num = DeadLink.objects.count()
    return render_to_response('smra_portal/search_results.html',
                            {'descform':desc_form, # we want this to be blank?
                             'metadataforms':schema_forms,    
                             'mediaform':media_form,
                             'resultsperpage':results_per_page, 
                             'resultsform': results_form,
                             'resultcount': paginator.count,
                             'resultsformdata': results_form_data, 
                             'origquery': desc_query,   # so that pagination works                     
                             'search_results':objs,
                             'deadlinks_num':deadlinks_num,
                             'historylist':historylisttuple,
                             'virts': virtlisttuple,
                             'currentvirt': currentvirt,
                             'subquery_formdata': subquery_formdata,      
                             'subquery_form': subquery_form,                           
                             'error_message':error_message}, 
                            context_instance=RequestContext(request))

def _truncate(value, arg):
    """
    Truncates a string after a given number of chars  
    Argument: Number of chars to truncate after
    """
    try:
        length = int(arg)
    except ValueError: # invalid literal for int()
        return value # Fail silently.
    if not isinstance(value, basestring):
        value = str(value)
    if (len(value) > length):
        return value[:length] + "..."
    else:
        return value

def _populate_metadata_forms(query):
    _make_metadata_forms(query=query)
    
def _make_metadata_forms(**kwargs):
    """ Returns a set of metadata perform_search forms"""
    fieldmap = {}
    schema_forms = []
    
    # fields in query expression are used to populate the initial values
    # of the form fields (usually in GET)
    
    if 'disj' in kwargs and kwargs['disj']:
        logger.debug("disj= %s" % kwargs['disj'])
                
        for qe in kwargs['disj']:
   
            for sch, field_type, field, operator, value, neg in [(x.schema, x.type,
                                           x.field, x.operator, x.value, x.negate)
                                 for x in qe.search_field_ops]:
                logger.debug("field=%s operator=%s value=%s neg=%s type=%s" % (field,
                                                                operator, 
                                                                   value,
                                                                   neg,
                                                                field_type))                    
                if sch:
                    paramname = str(sch[0]).lower() + "/" + field
                else:
                    paramname = str(field)                
                
                logger.debug("paramname=%s" % paramname)
                fieldmap[paramname] = value
            
                if neg:                
                    fieldmap["NOT/" + paramname] = True               
        
    schemas_for_search = []
    for ns,name in settings.SEARCH_SCHEMAS:
        logger.debug("ns=%s name=%s" % (ns,name))
        pks = [sch.pk for sch in Schema.objects.filter(namespace=ns,name=name)]
        logger.debug("pks=%s" % pks)
        schemas_for_search.extend(pks)
        
    logger.debug("schemas_for_search=%s" % schemas_for_search)
        
    # TODO: fix order of search schemas based on setting.
    
    for sch in Schema.objects.filter(pk__in=schemas_for_search):
        schema_form = SortedDict()
        for param in ParameterName.objects.filter(schema=sch):
            for _ in range(0,1):
                
                logger.debug("param.name=%s" % param.name)
                logger.debug(param)
                            
                # Each part of field has whitespace replaced with _ to 
                # make a complete identifier, which is required in grammer
                # of query expression
                if sch.name:  
                    field_name = sch.name + "/" + param.name.replace(' ','_').lower()
                else:
                    pass
                
                logger.debug("field_name = %s" % field_name)
                
                try:
                    initial_val = fieldmap[field_name]                    
                except KeyError:
                    logger.debug("keyerror:")
                    initial_val= ''
                logger.debug("initial_val=%s" % initial_val)
                
               
                # Each metadata form (e.g., film/director)
                # has an associated boolean field NOT/film/director.  If this is
                # set then we know we have negation.
                is_neg = False
                if "NOT/" + field_name in fieldmap:
                    is_neg = True
                
                field_label = "%s%s" % (param.name[0].upper(), 
                                        param.name[1:].lower())                
                    
                schema_form['NOT/%s' % field_name] = \
                    forms.BooleanField(required=False,label="AND NOT",initial=is_neg)
                
                #TODO: add proper help desks to each type of field
                if (param.type == ParameterName.NUMERIC):
                    schema_form[field_name] =  \
                        forms.IntegerField(required=False,initial=initial_val,label=field_label)
                elif (param.type == ParameterName.YEAR):
                    schema_form[field_name] =  \
                        forms.CharField(required=False,initial=initial_val,label=field_label)
                elif (param.type == ParameterName.DATE):
                    #schema_form[field_name] =  \
                    #    forms.DateField(required=False,initial=initial_val,label=field_label)
                    #FIXME: use a proper DateField, as it give error feedback
                    schema_form[field_name] =  \
                        forms.CharField(required=False,initial=initial_val,label=field_label)
                else:
                    schema_form[field_name] = \
                        forms.CharField(required=False,initial=initial_val,label=field_label)
                # TODO: handle LINKS type
   
        SchemaForm = type('SchemaForm', (forms.BaseForm,), {'base_fields':schema_form})
        
        # if used in POST, then want to take our new form and populate 
        # it the request values
        if 'req' in kwargs:
            schema = SchemaForm(kwargs['req'])
        else:
            schema = SchemaForm()
        
        schema_forms.append((sch.namespace,sch.name,sch.description,len(schema_form),schema))
       
       
    return schema_forms
      
      
def _make_desc_form(**kwargs):
    
    desc_query = []
    
    
    if 'disj' in kwargs and kwargs['disj']:      
        for qe in kwargs['disj']:
            for sch, fieldtype, field, value,neg in [(x.schema, x.type,
                                               x.field, x.value, x.negate)
                                     for x in qe.search_field_ops]:
                    logger.debug("field=%s value=%s neg=%s type=%s" % (field,
                                                                       value,
                                                                       neg, 
                                                                       fieldtype))
                            
                    if not sch and field.lower() == DESCLABEL:
                        if neg:                
                            desc_query.append('~%s%s"%s"' % (DESCLABEL,                
                                                   CONTAINS_OP,                
                                                   value))                
                        else:                
                            desc_query.append('%s%s"%s"'%(DESCLABEL,
                                                              CONTAINS_OP,
                                                              value))
                    
    desc_query_str = ''
    if desc_query:
        desc_query_str = 'VC:'
           
    desc_query_str += ' & '.join(desc_query)
    logger.debug("desc_query_str = %s" % desc_query_str)
    desc_form = DescForm({'desc_field':desc_query_str})
    
    return desc_form
  
  
def _make_media_form(**kwargs):
    """ Returns a form for specifying media to be searched"""
     
    #schema_names = []
    media_field = {}
#            
#    for sch in Schema.objects.all():
#        schema_names.append((sch.name, sch.description))        
#        
    schema_names = []
    for ns,name in settings.SEARCH_SCHEMAS:
        logger.debug("ns=%s name=%s" % (ns,name))
        try:
            sch = Schema.objects.get(namespace=ns,name=name)
        except Schema.DoesNotExist:
            logger.info("unknown schema %s/%s " % (ns,name) )
            pass
        else:   
            schema_names.append((sch.name,sch.description))        
    logger.debug("schema_names=%s" % schema_names)
        
    initial_choices = []
    not_initial_choices = []
    
    if 'disj' in kwargs and kwargs['disj']:
        logger.debug("disj= %s" % kwargs['disj'])
            
        for qe in kwargs['disj']:       

            for sch, field_type, field, value, neg in [(x.schema, x.type,
                                       x.field, x.value, x.negate)
                             for x in qe.search_field_ops]:
                                     
                logger.debug("field=%s value=%s neg=%s type=%s" % (field,
                                                                   value,
                                                                   neg, 
                                                                   field_type))
                        
                if not sch and not neg and field.lower() == 'mediatype':
                    initial_choices.append(value)
                      
                if not sch and neg and field.lower() == 'mediatype':
                    not_initial_choices.append(value)
                                                            
                
    logger.debug("intial_choices= %s" % initial_choices)
    logger.debug("not_intial_choices= %s" % not_initial_choices)
    multifield = forms.MultipleChoiceField(required=False,
                                           choices=[(name,"IS %s" % desc) 
                                                    for (name,desc)
                                                     in schema_names],
                                           label="AND",                                           
                                           widget=forms.SelectMultiple(attrs={'size':str(len(schema_names))})) 
       
    media_field['mediachoices'] = multifield
    
    not_multifield = forms.MultipleChoiceField(required=False,
                                           choices=[(name,"IS NOT %s" % desc) 
                                                    for (name,desc)
                                                     in schema_names],
                                            label="AND",
                                               widget=forms.SelectMultiple(attrs={'size':str(len(schema_names))})) 
       
    media_field['notmediachoices'] = not_multifield
    
    
    MediaForm = type('MediaForm', (forms.BaseForm,), {'base_fields':media_field})
    if 'req' in kwargs:
        mediaform = MediaForm(kwargs['req'], 
                            initial={'mediachoices':initial_choices,
                                     'notmediachoices':not_initial_choices})
    else:
        mediaform = MediaForm(initial={'mediachoices':initial_choices,
                                     'notmediachoices':not_initial_choices})
    
        
    return mediaform



def _make_results_form(**kwargs):
    """ Returns a form for specifying media to be searched"""        
    results_form_inc = SortedDict()         
    results_form_not = SortedDict()
    results_form_data = []
    result_count = 0
    if 'objs' in kwargs:
        for obj in kwargs['objs']:
            results_form_data.append( (obj.description,'%s' % obj.pk))                                    
            #results_form_data['%s' % obj.pk] = (obj.description,'%s' % obj.pk)          
            results_form_inc['SR/%s/INC' % obj.pk] = \
                    forms.BooleanField(required=False, label="ADD", initial='')
            results_form_not['SR/%s/NOT' % obj.pk] = \
                    forms.BooleanField(required=False, label="NOT", initial='')
            result_count += 1
            
    #http://www.b-list.org/weblog/2008/nov/09/dynamic-forms/ 
    ResultsForm = type('SchemaForm', (forms.BaseForm,),
                        {'base_fields':dict(results_form_inc.items() 
                                            + results_form_not.items())})
    
    #TODO: handle the initial values
    if 'req' in kwargs:
        resultsform = ResultsForm(kwargs['req'], 
                            initial={})
    else:
        resultsform = ResultsForm(initial={})
    logger.debug("results_form_inc = %s" % results_form_inc)       
    res = zip(results_form_inc.keys(),
                                results_form_not.keys(),results_form_data) 
    logger.debug("res = %s" % res)       
    return (resultsform,res,result_count)


    
def _make_subquery_form(**kwargs):
    subquery_form = SortedDict()
    subquery_form_del = SortedDict()
    subquery_form_inc = SortedDict()
    subquery_form_data = []
    
    from collections import defaultdict
    previous_value = defaultdict(bool)
    previous_not_value = defaultdict(bool)
  
    if 'disj' in kwargs and kwargs['disj']:
        logger.debug("disj= %s" % kwargs['disj'])
                
        for qe in kwargs['disj']: 
            for sch, field_type, field, value, neg in [(x.schema, x.type,
                                       x.field, x.value, x.negate)
                             for x in qe.search_field_ops]:
                logger.debug("field=%s value=%s neg=%s type=%s" % (field,
                                                                   value,
                                                                   neg, 
                                                                   field_type))
                        
                if not sch and not neg and field.lower() == SUBQUERYLABEL:
                    previous_value[value] = True
                      
                if not sch and neg and field.lower() == SUBQUERYLABEL:
                    previous_not_value[value] = True                          
  
    logger.debug("previous_value= %s" % repr(previous_value))
    logger.debug("previous_not_value=%s" % repr(previous_not_value))
  
    for obj in kwargs['objs']:
    
        # subquery_form_data has key suffix and holds labels and other
        # useful information for the field in its value tuple.  
        # subquery_form holds the actual form field.  By traversing 
        # keys of subquery_form_data in the template we can retrieve its 
        # data and index to get the equivalent form field as well.
                
        subquery_label = "%s" % obj.query
        logger.debug("subquery_label = %s" % subquery_label)
              
        subquery_form_data.append((obj.name,
                                             subquery_label,
                                             subquery_label[5:-2],
                                             obj.pk))
    
        # The key prefix identifies the various fields in this subquery entry
        # FIXME: make separate form for each line (like for metadata fields)         
        subquery_form_del['SQ/%s/DEL' % obj.pk] = forms.BooleanField(required=False,
                                     label="DEL")    
        #   subquery_form['SQ/%s/KEY' % obj.pk] = forms.BooleanField(required=False,
        #                                label=subquery_label)  
        logger.debug("initial{%s} = %s" % ('%s' % obj.pk,
                                           previous_value['%s' % obj.pk]))  
        subquery_form_inc['SQ/%s/INC' % obj.pk] = forms.BooleanField(required=False,
                                     label="OR",
                                     initial=previous_value['%s' % obj.pk])
    
    
        
    SubqueryForm = type('SubqueryForm', (forms.BaseForm,), 
                        {'base_fields':dict(subquery_form_inc.items() 
                                            + subquery_form_del.items())})
    
    logger.debug("subqueryform=%s" % SubqueryForm)
    
    #TODO: handle the initial values
    if 'req' in kwargs:
        logger.debug("req=%s" % kwargs['req'])        
        
        subqueryform = SubqueryForm(kwargs['req'])   
        logger.debug("subqueryform=%s" % subqueryform)        
      
    else:     
        subqueryform = SubqueryForm()           
    
    res = zip(subquery_form_inc.keys(),
              subquery_form_del.keys(),
              subquery_form_data)
    
    return (subqueryform,res)    
    
    

def collections(request):
    """ View all public collections
    
        :param request: the web request
        :type request: :class:`django.http.HttpRequest`
    """                
    virts = VirtualCollection.objects.filter(public=True)    
    return render_to_response(
                'smra_portal/public.html', 
                {'virts': virts}, 
                context_instance=RequestContext(request))     


def my_collections(request):
    """ View the virtual collections owned by the user
        
        :param request: the web request
        :type request: :class:`django.http.HttpRequest`
    """
    virts = VirtualCollection.objects.none()   
    virtform = SmallVirtForm()    
    if request.user.is_authenticated():
        
        profile = _retrieve_user_profile(request)
        #try:
        #    profile = request.user.get_profile()       
        #except UserProfile.DoesNotExist:
        #    profile = UserProfile(user=request.user,
        #                  results_per_page=settings.DEFAULT_RESULTS_PER_PAGE)
        #    profile.save()            
        virts = VirtualCollection.objects.filter(owned_by__exact=profile)
        
        if not virts:
            try:    
                logger.debug("%s %s" % (settings.VIRT_META_SCHEMA_NAMESPACE,
                                            settings.VIRT_META_SCHEMA_NAME))
                sch = Schema.objects.get(namespace
                                         =settings.VIRT_META_SCHEMA_NAMESPACE,
                                         name=settings.VIRT_META_SCHEMA_NAME)                
            except Schema.DoesNotExist:                                                                                                                    
                return HttpResponseServerError(
                                    stringbank.config['VIRT_SCHEMAS_MISSING'])              
            defaultvirt = VirtualCollection(
                                    owned_by=profile,
                                    name="Unnamed Collection",
                                description="Define and save this collection",
                                        key = "")
            defaultvirt.save()
            vcps = VirtualCollectionParameterSet(schema=sch,virt=defaultvirt)                
            vcps.ranking=100
            vcps.save()      
            for pn in ParameterName.objects.filter(schema=sch):                                    
                vcp = VirtualCollectionParameter(name=pn,
                                                     paramset=vcps,
                                                     value=pn.initial)
                vcp.save()
            harvest_record = VirtualCollectionHarvest(virt=defaultvirt)
            harvest_record.save()
            virts = VirtualCollection.objects.filter(owned_by__exact=profile)
            messages.success(request,"Initial Private Collection Created")
               
              
        error_message = None    
        if request.method == 'POST':        
            virtform = SmallVirtForm(request.POST)            
            if virtform.is_valid():
                # FIXME: Need to create this schema is not found
                logger.debug("%s" % Schema.objects.all())
                try:    
                    logger.debug("%s %s" % (settings.VIRT_META_SCHEMA_NAMESPACE,
                                            settings.VIRT_META_SCHEMA_NAME))
                    sch = Schema.objects.get(namespace
                                             =settings.VIRT_META_SCHEMA_NAMESPACE,
                                         name=settings.VIRT_META_SCHEMA_NAME)                
                except Schema.DoesNotExist:                                                                                                                    
                    return HttpResponseServerError(
                                            stringbank.config['VIRT_SCHEMAS_MISSING'])
                # TODO: make this whole task a single transaction
                newvirt = virtform.save(commit=False)
                newvirt.owned_by = profile
                # FIXME: use pk for the key for virtualcollection, though
                # it should be the handle or doi
                newvirt.key = ""
                newvirt.save()            
                vcps = VirtualCollectionParameterSet(schema=sch,virt=newvirt)                
                vcps.ranking=100
                vcps.save()                
                for pn in ParameterName.objects.filter(schema=sch):                                    
                    vcp = VirtualCollectionParameter(name=pn,
                                                     paramset=vcps,
                                                     value=pn.initial)
                    vcp.save()
                harvest_record = VirtualCollectionHarvest(virt = newvirt)
                harvest_record.save()          
            else:
                messages.error(request,"Invalid form data %s" % virtform.errors)                                   
            return redirect(reverse('virts'))
            
        else:
            virtform = SmallVirtForm()
    else:
        raise Http403  
        
    return render_to_response(
                'smra_portal/virts.html', 
                {'virts': virts,
                 'virtform':virtform,                
                'error_message':None}, 
                context_instance=RequestContext(request))


def _retrieve_user_profile(request):
    try:
        profile = request.user.get_profile()       
    except UserProfile.DoesNotExist:
        profile = UserProfile(
                      user=request.user,
                      results_per_page=settings.DEFAULT_RESULTS_PER_PAGE)      
        profile.save()
        my_party_record = PartyRecord(type="person",
                                      name=profile.user.get_full_name())
        my_party_record.save()
        
    return profile


def update_virt(request, virt_id):
    """ The search page for both authenticated and anonymous users.  
        Processes both POST to process changes to subquery sets and
        virtual collection and go generate disjunction and then GET
        to produce results lists 
         
        :param request: the web request
        :type request: :class:`django.http.HttpRequest`
        :param virt_id: the unique primary key of the virtual collection. If 0, then process the anonymous collection
        :type virt_id: primary key
    """
    virts = []
    subqueries = []
    profile = None    
    currentvirt = None
    # Try to validate, and load virtual collections and subquery
    if request.user.is_authenticated():
        profile = _retrieve_user_profile(request)
        virts = VirtualCollection.objects.filter(owned_by__exact=profile)
        subqueries = Subquery.objects.filter(part_of=virt_id,
                                             owned_by__exact=profile)
        try:
            virt_id_int = int(virt_id)
        except ValueError,TypeError:
            virt_id_int = 0
        
        if virt_id_int > 0:
            try:
                currentvirt = VirtualCollection.objects.get(owned_by__exact=profile,
                                                            pk=virt_id)
            except ObjectDoesNotExist:
                logger.debug("not found %s %s" % (profile, virt_id))
                raise Http404          
        else:
            currentvirt = None 
    else:
        currentvirt = None
    historylist = [] 
    # Check session data, and load any existing history
    if request.session.has_key('history'):          
        historylist = list(simplejson.loads(request.session['history']))
    else:
        historylist = []
      
    if request.method == 'POST':        
        return _handle_search_post(request, historylist, profile, 
                                   virts, subqueries, currentvirt)           
    if request.method == 'GET':
        return _handle_search_get(request, historylist, profile,
                                  virts, subqueries, currentvirt)     
    # User my try DELETE,CREATE or some invalid request
    return HttpResponseBadRequest()    
        
        
def _make_queryset(virt_id):
    virt = get_object_or_404(VirtualCollection, pk=virt_id)
    logger.debug("virt=" + virt.description)
    parameter_sets = VirtualCollectionParameterSet.objects.filter(virt=virt)

    logger.debug("virt parameter_sets=" + repr(parameter_sets))
    return VirtualCollectionParameter.objects.filter(
                                                 paramset__in=parameter_sets)
        
def _make_virt_form(virt_id,req):
    """
    Returns a VirtualCollectionParameter form for the virtual collection
    virt_id and populated by req if not None
    """            
    virt_form = SortedDict() # VirtsForm gets same ordering as virt_form_data
    virt_form_data = []    
    for virtparam in _make_queryset(virt_id):
        logger.debug("virtparam=%s" % virtparam)
        virt_form_data.append((virtparam.name.name,virtparam.name.help_text))
        #TODO: handle numeric type
        if virtparam.name.type == ParameterName.NUMERIC:
            initial_data = virtparam.value  
            virt_form['VC/%s' % virtparam.pk] = \
            forms.CharField(required=False,label="0",initial=initial_data)
        elif virtparam.name.type == ParameterName.YEAR:
            initial_data = virtparam.value  
            virt_form['VC/%s' % virtparam.pk] = \
            forms.CharField(required=False,label="0",initial=initial_data)
        elif virtparam.name.type == ParameterName.DATE:
            initial_data = virtparam.value  
            virt_form['VC/%s' % virtparam.pk] = \
            forms.DateField(required=False,label="0",initial=initial_data)
        elif virtparam.name.type == ParameterName.STRLIST:
            initial_data = virtparam.value  
    
            choices = [(virtparam.name.initial,
                        virtparam.name.initial)]
            try:
                choices = [(x,x) for x in 
                           list(ast.literal_eval(virtparam.name.choices))]                   
            except SyntaxError,e:
                logger.debug("virt.name.choices=%s" % virtparam.name.choices)       
                raise QueryExpressionError(e)
            except ValueError,e:
                logger.debug("virt.name.choices=%s" % virtparam.name.choices)
                raise QueryExpressionError(e)
            
            virt_form['VC/%s' % virtparam.pk] = forms.ChoiceField(
                                                        label="0",
                                                        choices=choices,
                                                        initial=initial_data)
        else:
            initial_data = virtparam.value  
            param_len = virtparam.name.max_length
            cols = (80 if param_len >= 80 else param_len)
            rows = param_len / cols
            logger.debug("param_len=%s" % param_len)
            logger.debug("cols=%s" % cols)
            logger.debug("rows=%s" % rows)
            
            virt_form['VC/%s' % virtparam.pk] = \
                forms.CharField(required=False,label="",initial=initial_data,
                            widget=forms.Textarea(
                                    attrs={'cols':cols,'rows':rows}))
        
                  
    logger.debug("virt_form = %s" % virt_form)
    logger.debug("virt_form_data = %s" % virt_form_data)
      
    VirtsForm = type('VirtForm',(forms.BaseForm,),{'base_fields':virt_form})        
    if req:        
        logger.debug("req = %s" % req )
        virtsform = VirtsForm(req)
    else:
        virtsform = VirtsForm() 
        logger.debug("blank")       
    for f in virtsform:
        logger.debug("f = %s" % f)        
    for f in virt_form_data:
        logger.debug("v = %s" % str(f))
    res = zip(virt_form_data, virtsform)
    logger.debug("res=%s" % res)
    return (virtsform,res)


def edit_virt_metadata(request, virt_id):
    """ View metadata for a virtual collection and process changes to metadata
    
        :param request: the web request
        :type request: :class:`django.http.HttpRequest`
        :param virt_id: the unique primary key of the virtual collection.
        :type virt_id: primary key
    """
    
    try:
        virt = VirtualCollection.objects.get(pk=virt_id)        
    except VirtualCollection.DoesNotExist:
        raise Http404
    try:
        vc_harvest = VirtualCollectionHarvest.objects.get(virt=virt)        
    except VirtualCollectionHarvest.DoesNotExist:
        vc_harvest = VirtualCollectionHarvest(virt=virt)
        vc_harvest.save()
    owner = False    
    profile = None
    if request.user.is_authenticated():      
        profile = _retrieve_user_profile(request)
        if virt.owned_by != profile: 
            if  not virt.public:
                raise Http403
            else:
                pass
        else:
            owner = True
        _handle_page_length(request,profile)
        results_per_page = profile.results_per_page
    else:
        results_per_page = settings.DEFAULT_RESULTS_PER_PAGE
        if not virt.public:
            raise Http403
    error_message = None
    logger.debug("handled page buttons")
    if request.method == 'POST' and ('makeprivate' in request.POST):
        if (request.user.is_authenticated 
            and virt.owned_by == profile
            and virt.public):
            virt.public = False
            virt.save() 
            messages.success(request,"Virtual Collection has been made private"
                             " and has been removed from the Feed for ANDS")                 
    if request.method == 'POST' and ('update' in request.POST):
        logger.debug("found POST update")             
        virt_desc_form = VirtForm(request.POST,instance=virt)
        if virt_desc_form.is_valid():
            update_virt = virt_desc_form.save()
            logger.debug("saving virt %s " % update_virt)
            messages.success(request,"Collection Updated")   
        else:
            logger.debug("invalid form data")
            messages.error(request,"Invalid form data")   
        
        (form, form_data)= _make_virt_form(virt_id,request.POST)        
        if form.is_valid():           
            if (request.user.is_authenticated 
            and virt.owned_by == profile):                     
                cleaned = form.cleaned_data #TODO
                logger.debug("found update")
                elems_changed = []
                for field in cleaned:
                    logger.debug("key=%s val=%s" % (field,cleaned[field]))                            
                    fieldkey_parts = field.split('/')
                    if (len(fieldkey_parts) == 2 and fieldkey_parts[0] == 'VC'):
                        key_str = fieldkey_parts[1]
                        logger.debug("key_str = %s" % key_str)                    
                        try:
                            key = int(key_str) 
                        except ValueError:
                            key = 0                                                        
                        val = cleaned[field]
                        logger.debug("key= %s val=%s" % (key,val))                    
                        param = None                                                    
                        try:                              
                            param = VirtualCollectionParameter.objects.get(pk=key,
                                                                           paramset__virt__owned_by=profile)    
                        except VirtualCollectionParameter.DoesNotExist:
                            logger.debug("403")
                            return Http403()  
                        logger.debug("param=%s" % param)            
                        if param.name.type == ParameterName.NUMERIC:
                            try:
                                value = int(val) 
                            except ValueError:
                                value = 0
                            if param.value != value:                                
                                param.value = value               
                                param.save()                 
                                elems_changed.append(param.name.name)                                                                                 
                        elif param.name.type == ParameterName.DATE:
                            try:
                                val = date(val)
                            except ValueError:
                                val = None
                            if param.value != val:
                                param.value = val
                                param.save()
                                elems_changed.append(param.name.name) 
                        elif param.name.type == ParameterName.YEAR:
                            try:
                                val = int(val)
                            except ValueError:
                                val = 0
                            if param.value != val:
                                param.value = val
                                param.save()
                                elems_changed.append(param.name.name) 
                        else:
                            if param.value != val:                                
                                param.value = val                                                                                                                                                          
                                param.save()
                                elems_changed.append(param.name.name)  
                                              
                if elems_changed:
                    virt.update_time = datetime.datetime.now()
                    virt.save()
                    messages.success(request,"Metadata fields: %s for Virtual Collection updated" % ','.join(elems_changed))
                  
            else:
                logger.debug("No permission")
                messages.error(request,"You do not have permission to change this item")                                        
        else:
            logger.debug("invalid form data")
            messages.error(request,"Invalid form data")     
    else:        
        logger.debug("not POST or update")                
        (form,form_data) = _make_virt_form(virt_id,None)  
        virt_desc_form = VirtForm(instance=virt)   
        
    logger.debug("done meta")
        
    desc_query = virt.definition
    if not desc_query:
        # brand new virt
        desc_query=str(["VC:"])            
    disjunction = Disjunction()
    objs = MediaObject.objects.none()
    try:   
        disjunction = Disjunction(parse_disjunction(desc_query))
        logger.debug("disjunction is %s" % str(disjunction))
        objs = disjunction.get_objects_for_disj(virt)            
    except QueryExpressionError:            
        messages.error(request,"There was an error in your query")
        error_message = True                     
         
    
    page = _handle_page_buttons(request)
    
    if profile:
        paginator = Paginator(objs,profile.results_per_page) # Show 25 contacts per page
    else:
        paginator = Paginator(objs,settings.DEFAULT_RESULTS_PER_PAGE) # Show 25 contacts per page
    
    try:
        objs = paginator.page(page)
    except (PageNotAnInteger,TypeError):
        # If page is not an integer, deliver first page.
        objs = paginator.page(1)
    except EmptyPage:
        # If page is out of range (e.g. 9999), deliver last page of results.
        objs = paginator.page(paginator.num_pages)
     
                                    
    return render_to_response(
                'smra_portal/virt_meta.html', 
                {'currentvirt': virt,
                 'virtformdata':form_data,
                 'virtform':form, 
                 'virtdescform':virt_desc_form,
                 'resultsperpage': results_per_page,
                 'virtobjects': objs,
                 'resultcount': paginator.count,
                 'harvest':vc_harvest,
                 'owner':owner,           
                'error_message':error_message}, 
                context_instance=RequestContext(request))        
        

def media_object_list(request): 
    """ View the list of all media objects in the database
        
        :param request: the web request
        :type request: :class:`django.http.HttpRequest`
    """
    
    results_per_page = settings.DEFAULT_RESULTS_PER_PAGE
    
    if request.user.is_authenticated():
        profile = _retrieve_user_profile(request)
        results_per_page = profile.results_per_page
        
    
#        if 'pagelength' in request.GET:
#            plen = request.GET['pagelength']
#            try:
#                pagelen = int(plen)
#            except ValueError:
#                pagelen = settings.DEFAULT_RESULTS_PER_PAGE
#            if pagelen <= 0:
#                pagelen = settings.DEFAULT_RESULTS_PER_PAGE
#            profile.results_per_page = pagelen
#            profile.save()
          
                        
    
#    if profile:
#        results_per_page = profile.results_per_page
#    else:
#        results_per_page = settings.DEFAULT_RESULTS_PER_PAGE 
                     
    media_objects = MediaObject.objects.all()
    paginator = Paginator(media_objects,results_per_page) 
        
    
    
    page = request.GET.get('page')
    try:
        mo = paginator.page(page)
    except (PageNotAnInteger,TypeError):
        # If page is not an integer, deliver first page.
        mo = paginator.page(1)
    except EmptyPage:
        # If page is out of range (e.g. 9999), deliver last page of results.
        mo = paginator.page(paginator.num_pages)
    
    return render_to_response('smra_portal/mediaobjects.html', 
                              {"mediaobjects": mo},
                              context_instance=RequestContext(request))

# 
# 
#class MediaObjectListView(ListView):
#    """ Displays all the media_objects in the db"""
#
#    model = MediaObject
#    template_name = 'smra_portal/mediaobjects.html'
#    paginate_by = 20
#
#    def get_context_data(self, **kwargs):
#        # Call the base implementation first to get a context
#        context = super(MediaObjectListView, self).get_context_data(**kwargs)
#        context['deadlinks_num'] = DeadLink.objects.count()
#        return context
    
       
class SMRAAboutView(TemplateView):
    """View the about page for the application"""
    template_name = "smra_portal/about.html"
    
    def get_context_data(self, **kwargs):
        # Call the base implementation first to get a context
        context = super(SMRAAboutView, self).get_context_data(**kwargs)
        context['deadlinks_num'] = DeadLink.objects.count()
        return context
    
      
import urllib2
        
WAITTIME = 10

def _try_link(link):
    """ Tries to open link and returns error messasge if it failed, or
    None otherwise"""
    responses = {
    100: ('Continue', 'Request received, please continue'),
    101: ('Switching Protocols',
          'Switching to new protocol; obey Upgrade header'),

    200: ('OK', 'Request fulfilled, document follows'),
    201: ('Created', 'Document created, URL follows'),
    202: ('Accepted',
          'Request accepted, processing continues off-line'),
    203: ('Non-Authoritative Information', 'Request fulfilled from cache'),
    204: ('No Content', 'Request fulfilled, nothing follows'),
    205: ('Reset Content', 'Clear input form for further input.'),
    206: ('Partial Content', 'Partial content follows.'),

    300: ('Multiple Choices',
          'Object has several resources -- see URI list'),
    301: ('Moved Permanently', 'Object moved permanently -- see URI list'),
    302: ('Found', 'Object moved temporarily -- see URI list'),
    303: ('See Other', 'Object moved -- see Method and URL list'),
    304: ('Not Modified',
          'Document has not changed since given time'),
    305: ('Use Proxy',
          'You must use proxy specified in Location to access this '
          'resource.'),
    307: ('Temporary Redirect',
          'Object moved temporarily -- see URI list'),

    400: ('Bad Request',
          'Bad request syntax or unsupported method'),
    401: ('Unauthorized',
          'No permission -- see authorization schemes'),
    402: ('Payment Required',
          'No payment -- see charging schemes'),
    403: ('Forbidden',
          'Request forbidden -- authorization will not help'),
    404: ('Not Found', 'Nothing matches the given URI'),
    405: ('Method Not Allowed',
          'Specified method is invalid for this server.'),
    406: ('Not Acceptable', 'URI not available in preferred format.'),
    407: ('Proxy Authentication Required', 'You must authenticate with '
          'this proxy before proceeding.'),
    408: ('Request Timeout', 'Request timed out; try again later.'),
    409: ('Conflict', 'Request conflict.'),
    410: ('Gone',
          'URI no longer exists and has been permanently removed.'),
    411: ('Length Required', 'Client must specify Content-Length.'),
    412: ('Precondition Failed', 'Precondition in headers is false.'),
    413: ('Request Entity Too Large', 'Entity is too large.'),
    414: ('Request-URI Too Long', 'URI is too long.'),
    415: ('Unsupported Media Type', 'Entity body in unsupported format.'),
    416: ('Requested Range Not Satisfiable',
          'Cannot satisfy request range.'),
    417: ('Expectation Failed',
          'Expect condition could not be satisfied.'),

    500: ('Internal Server Error', 'Server got itself in trouble'),
    501: ('Not Implemented',
          'Server does not support this operation'),
    502: ('Bad Gateway', 'Invalid responses from another server/proxy.'),
    503: ('Service Unavailable',
          'The server cannot process the request due to a high load'),
    504: ('Gateway Timeout',
          'The gateway server did not receive a timely response'),
    505: ('HTTP Version Not Supported', 'Cannot fulfill request.'),
    }
    error_message = None
    
    #Assume that if link does not have HTTP or FTP prefix, we mean't HTTP
    if link.lower().find('http://') < 0 and link.lower().find('ftp://') < 0:
        link = "http://%s" % link
     
    time.sleep(WAITTIME)  
    req = Request(link)                
    try:
        _ = urllib2.urlopen(req, timeout=60)
    except HTTPError, err:
        error_message = "HTTPError: %s %s" % (responses[err.code], err.code)
    except URLError, err:
        error_message = "URLError: %s" % err.reason
    except ValueError, err:
        error_message = "ValueError: %s" % err
    
    return error_message 


def make_link_check():
    """ Check all existing DeadLinks and MediaObjectParamters with 
    LINK value to make sure that link works.  Can only be run
    as super user for time being"""
                            
    dead_links = DeadLink.objects.all()
    checked = [] 
    for dead_link in dead_links:
        logger.debug("existing param = %s" % dead_link.parameter.value)
        error_message = _try_link(dead_link.parameter.value)
        if not error_message:           
            dead_link.delete()
        else:
            dead_link.save() # so that last access is updated
        checked.append(dead_link.parameter)                    
        
    params = MediaObjectParameter.objects.filter(name__type
                                                 =ParameterName.LINK)
    for param in params:
        if not param in checked:
            logger.debug("param = %s" % param)   
            error_message = _try_link(param.value)
       
            if error_message:
                logger.debug("error_message=%s" % error_message)
                dead_link = DeadLink(parameter=param, reason=error_message)
                dead_link.save()
   
def checklinks(request):
    """ View any known Deadlinks, but only for the admin user
    
        :param request: the web request
        :type request: :class:`django.http.HttpRequest`
    """
    
    if request.user.is_authenticated() and request.user.is_superuser:
        dead_links = DeadLink.objects.all()
        deadlinks_num = DeadLink.objects.count()
        return render_to_response('smra_portal/deadlinks.html',
                        {'error_message':'', 
                         'deadlinks_num':deadlinks_num,
                         'dead_links':dead_links}, 
                        context_instance=RequestContext(request))
    return redirect(reverse('virts'))


   
def test(request):
    """ View a simple test page"""
    return render_to_response('smra_portal/test.html',
                      
                    context_instance=RequestContext(request))
    
def publish(request,virt_id):
    """ Process the publishing of a collection through RIFCS 
        to external entites, including associating party records
        
        :param request: the web request
        :type request: :class:`django.http.HttpRequest`
        :param virt_id: the unique primary key of the virtual collection.
        :type virt_id: primary key
        """
    
    # Try to validate, and load virtual collections and subquery
    if request.user.is_authenticated():
        
        profile = _retrieve_user_profile(request)
        
        try:
            currentvirt = VirtualCollection.objects.get(owned_by__exact=profile,
                                          pk=virt_id)
        except ObjectDoesNotExist:
            raise Http404           
    else:
        return HttpResponseForbidden()
    
    from smra.smra_portal.publish.ANDSService import ANDSService
    
    publishService = ANDSService(currentvirt)
    
    if request.method == "POST":
        
       
        context_dict = {}
        if "legal" in request.POST:
            currentvirt.public=True
            currentvirt.save()
            
            context_dict['publish_result'] = \
                publishService.execute_publishers(request)
            
            publish_failed = False
            for result in context_dict['publish_result']:
                if not result['status']:
                    messages.error(request,"problem with publish")
                    publish_failed = True
            if not publish_failed:
                messages.success(request,stringbank.success['COLLECTION_NOW_SCHEDULED'])
            return redirect(reverse('virts'))
        else:
            messages.error(request,stringbank.error['AGREE_TERMS'])
            
            return redirect(reverse('virtpublish',
                                args=[currentvirt.pk]))
            
    else:
        
        
        legalfilepath = os.path.join(os.path.dirname(__file__),
                               'publish/legal.txt').replace('\\', '/')
        try:
            legalfile = open(legalfilepath,"r")
        except IOError:
            raise Http403()
        
        text = legalfile.read()
        legalfile.close()
        context_dict= {}
        context_dict = dict()
        logger.debug("template paths = %s" % publishService.get_template_paths())
        cd = dict({'legal':text,
                               'currentvirt':currentvirt,
                               'username':request.user,                               
                               'publish_forms':publishService.get_template_paths()},
                               **publishService.get_contexts(request))
        logger.debug("cd=%s" % cd)
        return render_to_response('smra_portal/publish.html',
                              cd,
                              context_instance=RequestContext(request))
    
def rif_cs(request):
    """ Entry point for direct ingestion of virtual collections via RIF-CS
        by external entity, such as ANDS
        
        :param request: the web request
        :type request: :class:`django.http.HttpRequest`
        
        """
    virts = VirtualCollectionHarvest.objects.filter(virt__public=True)
    
    
    # Assume that ingesting always works for the direct pull method at ANDS
    vc = []
    for virt in virts:
        virt.ingested_time = datetime.datetime.now()
        virt.save()
        vc.append(virt.virt)
        
          
    try:
        parties = PartyRecord.objects.all()
    except PartyRecord.DoesNotExist:
        parties = PartyRecord.objects.none()                

    #FIXME: this is only a heuristic, better to pull auth.user fields
    party_list = []
    for party in parties:
        part = party.name.split(" ")
        last = part[-1]
        first = "X".join(part[:-1])
        party_list.append([first, last, party.type, party.pk,party])
        
    return render_to_response('rif_cs_profile/rif-cs.xml',
                           {'virts': vc,
                            'now': datetime.datetime.now(),
                            'parties': party_list,
                            'activity_rif_cs': None,}
                              ,context_instance=RequestContext(request),
                             mimetype='application/xml' )
     
#def mint_handle(request):
#    
#    handle=""
#    authType = ""
#    logger.debug("request is %s" % request)
#
#    
##    if request.method == 'GET':
##        if 'type' in request.GET:
##            logger.debug("type = %s" % request.GET['type'])
##                         
##        if 'value' in request.GET:
##            logger.debug("value = %s" % request.GET['value'])
##            handle=request.GET['value']
##                        
##        
##        from xml.dom import minidom
##        body = request
##        xmldoc = minidom.parseString(body)
##        
##        properties = xmldoc.getElementsByTagName('property')
##        for property in properties:
##            attributes = property.attributes
##            if attributes['name'] == 'authType':
##                authType = attributes['value']
##         
##            
##        #handle = xmldoc.firstChild.childNodes[1].attributes["handle"].value
##        authtype =  xmldoc.firstChild.childNodes[1].attributes["handle"].value
##        logger.debug("request=%s" % request)
#        
#    return render_to_response('mint.xml',
#                           {'authType':authType,
#                            'handleURL':handle}
#                              ,context_instance=RequestContext(request),
#                             mimetype='application/xml' )
#    


    